from django.template import RequestContext
from django.http import HttpResponse,HttpResponseRedirect
from django.http import HttpResponseForbidden
from django.core.urlresolvers import reverse
from django.shortcuts import redirect, render, get_object_or_404
from django.contrib import auth, messages
from django.contrib.auth.models import User, Group
from django.views.decorators.csrf import csrf_exempt
from disn.settings import YUNPIAN_APIKEY, PROJECT_PATH, BASE_DIR
from disn.settings import TOKEN, APPID, APPSECRET, ENCODING_AES_KEY
from disn.settings import redis_instance, get_wechat_client
from disn.utilities import *
import random, requests, json, os, pdfkit, time, datetime, urllib
from .models import *
from django.core import serializers
from django.contrib.auth.decorators import login_required
from account.weixin import get_jssdk_signature
from django.http import HttpResponseForbidden
from django.template.loader import render_to_string
from django.core.paginator import Paginator
from taskqueue.tasks import *
from lxml import etree
from urllib.parse import unquote, quote

import numpy as np
from bs4 import BeautifulSoup

header = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}

hds=[{'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},\
{'User-Agent':'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'},\
{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'}]

s = requests.Session()
a = requests.adapters.HTTPAdapter(max_retries=3)
b = requests.adapters.HTTPAdapter(max_retries=3)
s.mount('http://', a)
s.mount('https://', b)


@login_required
def import_chinese_word(request):
    url = 'http://www.qjmy.cn/media/filer_public/75/5a/755a71b3-496d-4a34-b4ef-39843d9a8600/chinese_word.txt'
    resp = s.get(url, headers=header)
    if resp.status_code == 200:
        print('Successfully fetched data from [%s]' %url)
    else:
        return HttpResponse('Please check server network connection and test URL[%s] in your browser.' % url)
    row_arr = resp.text.split('\r\n')
    for item in row_arr:
        if item and len(item) > 0:
            domain, created = Domain.objects.get_or_create(word=item.strip())
            if created:
                domain.category = '0'
                domain.save()
                print('Created:%s' %item)
    return HttpResponse('Done')


def book_spider(book_tag):
    page_num=0;
    book_list=[]
    try_times=0

    while(1):
        #url='http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0' # For Test
        url='http://www.douban.com/tag/'+quote(book_tag)+'/book?start='+str(page_num*15)
        time.sleep(np.random.rand()*5)

        #Last Version
        # try:
        #     req = urllib2.Request(url, headers=hds[page_num%len(hds)])
        #     source_code = hds[page_num%len(hds)].urlopen(req).read()
        #     plain_text=str(source_code)
        # except (urllib2.HTTPError, urllib2.URLError) as e:
        #     print(e)
        #     continue

        resp = s.get(url, headers=hds[page_num%len(hds)])
        source_code = resp.content
        plain_text = resp.text

        ##Previous Version, IP is easy to be Forbidden
        #source_code = requests.get(url)
        #plain_text = source_code.text

        soup = BeautifulSoup(plain_text)
        list_soup = soup.find('div', {'class': 'mod book-list'})

        try_times+=1;
        if list_soup==None and try_times<200:
            continue
        elif list_soup==None or len(list_soup)<=1:
            break # Break when no informatoin got after 200 times requesting

        for book_info in list_soup.findAll('dd'):
            title = book_info.find('a', {'class':'title'}).string.strip()
            desc = book_info.find('div', {'class':'desc'}).string.strip()
            desc_list = desc.split('/')
            book_url = book_info.find('a', {'class':'title'}).get('href')

            try:
                author_info = '作者/译者： ' + '/'.join(desc_list[0:-3])
            except:
                author_info ='作者/译者： 暂无'
            try:
                pub_info = '出版信息： ' + '/'.join(desc_list[-3:])
            except:
                pub_info = '出版信息： 暂无'
            try:
                rating = book_info.find('span', {'class':'rating_nums'}).string.strip()
            except:
                rating='0.0'
            try:
                #people_num = book_info.findAll('span')[2].string.strip()
                people_num = get_people_num(book_url)
                people_num = people_num.strip('人评价')
            except:
                people_num ='0'

            book_list.append([title,rating,people_num,author_info,pub_info])
            print(title,rating,people_num,author_info,pub_info)
            try_times=0 #set 0 when got valid information
        page_num+=1
        print('Downloading Information From Page %d' % page_num)
    return book_list

def get_people_num(url):
    #url='http://book.douban.com/subject/6082808/?from=tag_all' # For Test
    # try:
    #     req = urllib2.Request(url, headers=hds[np.random.randint(0,len(hds))])
    #     source_code = urllib2.urlopen(req).read()
    #     plain_text=str(source_code)
    # except (urllib2.HTTPError, urllib2.URLError) as e:
    #     print(e)

    resp = s.get(url, headers=hds[np.random.randint(0,len(hds))])
    source_code = resp.content
    plain_text = resp.text

    soup = BeautifulSoup(plain_text)
    people_num=soup.find('div',{'class':'rating_sum'}).findAll('span')[1].string.strip()
    return people_num

def do_spider(book_tag_lists):
    book_lists=[]
    for book_tag in book_tag_lists:
        book_list=book_spider(book_tag)
        book_list=sorted(book_list,key=lambda x:x[1],reverse=True)
        book_lists.append(book_list)
    return book_lists

@login_required
def import_douban_books(request):
    book_tag_lists = ['机器学习','互联网']
    book_lists=do_spider(book_tag_lists)
    for item in book_lists:
        print(item)
    return HttpResponse('Done')
